#!  /usr/bin/env pytreon
#ecoding=utf-8
import sys
import os
curPath = os.path.abspath(os.path.dirname(__file__))
rootPath = os.path.split(curPath)[0]
sys.path.append(rootPath)
from yuncaiji.dao.template_url import TemplateUrlDao
from yuncaiji.dao.template_url_queue import TemplateUrlQueueDao
from yuncaiji.spider.spider import Spider
from yuncaiji.schedule.template2cache import TemplateUrl2CacheSchedule
from threading import  Thread
from yuncaiji.setting import *
import  time

if __name__=='__main__':
    #?秒从mysql读一次,保存到redis
    thread=Thread(target=TemplateUrl2CacheSchedule.start,args=(READ_URL_TIMES,))
    thread.start()
    # print('---------------------对象池初始化---------------------')
    # SafeObjectPool.init();
    while(True):
        templateUrls=TemplateUrlDao.get_no_spider_urls_and_update(READ_URL_Max);
        if templateUrls and len(templateUrls)>0:
            print('从数据库中获取：'+str(len(templateUrls))+'个模板')
            #请求url 返回html,线程数固定，平均分配链接
            links_size=len(templateUrls)
            per_thread_links_size=int(links_size/SELENIUM_THREAD_COUNT)
            threads=[]
            if links_size<=SELENIUM_THREAD_COUNT:
                spider=Spider()
                t=Thread(target=spider.start_spider,args=(templateUrls,))
                threads.append(t)
            else:
                for i in range(0,SELENIUM_THREAD_COUNT):
                    spider=Spider()
                    start=int(i*per_thread_links_size)
                    if i==SELENIUM_THREAD_COUNT-1:
                        end=int(links_size)
                    else:
                        end=int(start+per_thread_links_size)
                    per_thread_links=list(templateUrls)[start:end]
                    t=Thread(target=spider.start_spider,args=(per_thread_links,))
                    threads.append(t)

            for t in threads:
                t.start()
            for t in threads:
                t.join()
            print("多线程执行完成,共请求 %s 个连接"%(str(links_size)))
            print("Redis队列中TemplateUrl总个数："+str(TemplateUrlQueueDao.qsize()))
        else:
            #存储过程，从数据库中获取：0 个模板
            print('没有要执行的模板')
            time.sleep(READ_URL_TIMES)
